#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author;Tsukasa

import json
from multiprocessing import Pool
import requests
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

ua = UserAgent()
headers1 = {'User-Agent': 'ua.ramdom'}
import re
import pandas as pd
import pymongo
import os

Mongo_Url = 'localhost'
Mongo_DB = 'Lianjia'
t_sub_area = 'sub_area'
t_xiaoqu = 'xiaoqu'
client = pymongo.MongoClient(Mongo_Url)
db = client[Mongo_DB]
base_url = "https://bj.lianjia.com"


def get_xiaoqu_info(xiaoqu_id):  # 分析详细url获取所需信息
    # if db[t_xiaoqu].find_one({"小区编号": xiaoqu_id}) is not None:
    #     return None
    xiaoqu_url = 'https://bj.lianjia.com/xiaoqu/' + xiaoqu_id
    res = requests.get(xiaoqu_url, 'lxml', headers=headers1)
    if res.status_code == 200:
        info = {}
        soup = BeautifulSoup(res.text, 'lxml')
        title = soup.find(class_='detailTitle')
        if title is None:
            return None
        info['小区名称'] = title.text
        desc = soup.find(class_='detailDesc')
        if desc is None:
            return None
        info['小区别称'] = desc.text
        info['小区编号'] = xiaoqu_id
        #       基本属性
        for i in soup.select('.xiaoquInfo .xiaoquInfoItem'):
            spans = i.find_all('span')
            if spans is not None and len(spans) == 2:
                info[spans[0].text] = spans[1].text.strip('\n').strip(' ').strip('\n')

        return info


def add_or_update_to_MongoDB(xiaoqu_info):
    if xiaoqu_info is None:
        return
    if db[t_xiaoqu].update_one({'小区编号': xiaoqu_info['小区编号']}, {'$set': xiaoqu_info}, True):
        print(xiaoqu_info)


def main(info):
    xiaoqu_info = get_xiaoqu_info(info['小区编号'])
    add_or_update_to_MongoDB(xiaoqu_info)


if __name__ == '__main__':
    user_in_city = 'bj'
    print('获取北京所有小区的信息')
    with Pool(os.cpu_count()) as pool:
        pool.map(main, [xiaoqu_info for xiaoqu_info in db[t_xiaoqu].find().batch_size(50) if
                        '小区名称' not in xiaoqu_info or xiaoqu_info['小区名称'] is None])
